const fs = require("fs");
const path = require("path");
const scrapy = require("node-scrapy");
const savedir = "./save";
if (!fs.existsSync(savedir)) {
    fs.mkdirSync(savedir);
}


//工具函数
let paths = [];

function getpath() {
    return path.join(savedir, ...paths);
}

function getfilename(fname) {
    return path.join(getpath(), fname);
}
/**
 * 保证paths表示的目录存在
 */
function ensure_paths() {
    if (!fs.existsSync(getpath())) {
        fs.mkdirSync(getpath());
    }
}
//确保子目录存在
function ensure_dir(dir) {

    let p = path.join(getpath(), dir);
    if (!fs.existsSync(p)) {
        fs.mkdirSync(p);
    }
}

function enter_dir(name) {
    //防止错误字符
    name = name.replace(/[\/"\\:*?><|]/g, " ");

    paths.push(name);
    ensure_paths();

}

function leave_dir() {
    paths.pop();
}

//下载部分
const req = require("request");
let fail_num = 1;
async function downloadAndReturn(url) {
    return new Promise((resolve, reject) => {
        req.get(url, (err, res, body) => {
            if (res.statusCode == 200) {
                resolve(body);
            } else if (res.statusCode >= 400) {
                reject("错误");
            }
        })
    })
}

function download(url, fpath) {
    try {
        req.get(url, (err, res, body) => {
            if (err == null);
            //res.pipe(fs.createWriteStream(fpath));
            else {
                console.log(`${fail_num++}个文件下载失败!`);
            }
        }).pipe(fs.createWriteStream(fpath));
    } catch (e) {
        console.log(`${fail_num++}个文件下载失败!`);
    }

}

//同步化
async function scrape_async(url, model) {
    return new Promise((resolve, reject) => {
        scrapy.scrape(url, model, (err, data) => {
            if (err != null) reject(url.message);
            resolve(data);
        })
    })
}

function writefile(file, lst) {
    let str = "";
    lst.forEach(element => {
        str += `${element}\n`;
    });
    fs.writeFileSync(file, str);
}

function sleep(milliSeconds) {
    var startTime = new Date().getTime();
    while (new Date().getTime() < startTime + milliSeconds);
};



///开始抓取器代码

//模型定义


const url = "https://m.w3cschool.cn/tensorflow_python/tensorflow_python-bm7y28si.html";


async function parse(url, container_selector) {
    //得到是否有子菜单的数组
    let subs = await hasSubMenu(url, container_selector);
    for (let i = 0; i < subs.length; ++i) {
        let value = subs[i];
        let idx = i;
        //生成对应item的指定选择器
        const selector = `${container_selector}:nth-of-type(${idx+1})`;
        //对每个item 如果有子菜单 调用parseMenu1
        //如果没有调用parseMenu2
        if (value) {
            let ret = await parseMenu1(url, selector);
            //进入子菜单抓取
            //进入子目录
            enter_dir(ret.menu);
            //解析子菜单
            await parse(url, ret.subsele);
            //离开子目录
            leave_dir();
        } else {
            let ret = await parseMenu2(url, selector);
            //进入内容抓取程序
            await parseContent(ret.title, ret.url);

        }
    }
}
//生成guid
function guid() {
    function S4() {
       return (((1+Math.random())*0x10000)|0).toString(16).substring(1);
    }
    return (S4()+S4()+"-"+S4()+"-"+S4()+"-"+S4()+"-"+S4()+S4()+S4());
}

const cio = require("cheerio");
/**
 * 
 * @param {string} title 要抓取的内容页的标题（md文件名）
 * @param {string} url 要抓取的内容页的地址
 */
async function parseContent(title, url) {
    function getHtml(htmltext) {
        //提取html内容
        let $ = cio.load(htmltext);
        //删除不必要内容
        $("body mip-sidebar").remove();
        $("html script").remove();
        $("link").remove();
        $("meta").remove();
        $(".appdownload-nav").remove();
        $(".page.project-page").children("nav,header").remove();
        $(".row.prenext").remove();
        return $.html();
    }

    
    function localize(htmltext) {
        let $ = cio.load(htmltext);
        //搜索所有img
        let imgs = $("img");
        imgs.each((index,element)=>{
            let ele=$(element);
            let imgurl=ele.attr("src");
            //下载图片
            //生成图片名
            let name=`${guid()}.jpg`;
            let filename=getfilename(name);
            download(imgurl,filename);
            //替换链接
            ele.attr("src",`./${name}`);
        })
        return $.html();
    }

    //抓取内容页
    const model = {
        html: {
            selector: "html",
            get: "text"
        }
    }
    //抓取并保存到html文件
    let filepath = getfilename(`${title}.html`);
    let data = await scrape_async(url, model);
    //
    let text = await downloadAndReturn(url);
    fs.writeFileSync(filepath,localize(getHtml(text)));
}
async function parseMain(url) {
    let mainitems = ".project-block>.project-item";
    parse(url, mainitems);
}

/**
 * 从一系列project item中提取菜单项并生成子菜单容器的选择器
 * @param {string} url 
 * @param {string} container_selector 
 */
async function parseMenu1(url, container_selector) {
    //提取一级菜单的标题
    const menumodel = {
        menu1_text: `${container_selector}>div>.project-menu1`
    }
    let data = await scrape_async(url, menumodel);
    //生成子菜单items
    let subsele = `${container_selector}>div:nth-of-type(2)>.project-item`;
    return {
        menu: data.menu1_text,
        subsele
    };
}

/**
 * 从一系列project item中提取最后一级菜单（mip link的title和href）
 * @param {string} url 
 * @param {string} container_selector
 */
async function parseMenu2(url, container_selector) {
    const model = {
        title: {
            selector: `${container_selector} mip-link`,
            get: "title"
        },
        url: {
            selector: `${container_selector} mip-link`,
            get: "href"
        }
    };
    let data = await scrape_async(url, model);
    return {
        title: data.title,
        url: data.url
    };
}

/**
 * 从一系列project item中得到判断每个item是否有子菜单的bool列表
 * @param {string} url 
 * @param {string} container_selector 
 */
async function hasSubMenu(url, container_selector) {
    const model = {
        test: `${container_selector}`
    }
    let data = await scrape_async(url, model);
    let ret = []
    for (let i = 0; i < data.test.length; ++i) {
        //生成抓取第i+1个project item第模型
        const smodel = {
            sub: `${container_selector}:nth-of-type(${i+1})>div:nth-of-type(2)`
        };
        let subdata = await scrape_async(url, smodel);
        if (subdata.sub == null) {
            //没有子菜单
            ret[i] = false;
        } else ret[i] = true;
    }
    return ret;
}

parseMain(url);